package com.nlp.visualization.crawler.processor;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.nlp.visualization.common.CONSTANTS;
import com.nlp.visualization.utils.ConvertUtils;
import org.springframework.stereotype.Component;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.JsonPathSelector;

/**
 * @author LXM
 * @Title: ProcessorSina
 * @Description: 处理新浪新闻爬取内容的类
 * @date 2018/3/4下午6:55
 */
public class ProcessorSina extends Processor {
    private static String[] INIT_URL = CONSTANTS.SINA_CHANNELS;

    private static String sinaType_1 = "feed.mix";

    private static String sinaType_2 = "roll.new";

    private static String SINA = "sina";

    /**
     * 爬取当天最新的新闻
     *
     * @param page
     */
    public void processor(Page page) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        String strDate = sdf.format(new Date());
        String reg = "http://\\w+\\.sina\\.com\\.cn/(\\w+/)+" + strDate + "/doc-\\w+.shtml";
        if (page.getUrl().toString().indexOf(sinaType_1) != -1 || page.getUrl().toString().indexOf(sinaType_2) != -1) {
            try {
                initProcessor(page);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        } else if (page.getUrl().regex(reg).match()) {
            contentProcessor(page);
        } else {
            page.setSkip(true);
        }
    }

    /**
     * 爬取指定日期的新闻
     *
     * @param page
     * @param date
     */
    @Override
    public void processor(Page page, String date) {
        String reg = "http://\\w+\\.sina\\.com\\.cn/(\\w+/)+" + date + "/doc-\\w+.shtml";
        if (page.getUrl().toString().indexOf(sinaType_1) != -1 || page.getUrl().toString().indexOf(sinaType_2) != -1) {
            try {
                initProcessor(page, date);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
//        else if (page.getUrl().regex(reg).match()) {
//            contentProcessor(page);
//        }

        else if (page.getUrl().toString().indexOf(SINA) != -1) {
            contentProcessor(page);
        } else {
            page.setSkip(true);
        }
    }


    public void init(Spider spider) {
        Request[] request = new Request[INIT_URL.length];
        for (int i = 0; i < request.length; i++) {
            request[i] = new Request(INIT_URL[i]).putExtra("_charset", "gb2312")
                    .setPriority(1);
        }
        spider.addRequest(request);
    }


    /**
     * 筛选出今天最新的文章
     *
     * @param page
     * @throws ParseException
     */
    public void initProcessor(Page page) throws ParseException {

        List<String> links = new JsonPathSelector("$.list[*].url").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> tags = new JsonPathSelector("$.list[*].channel.title").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> titles = new JsonPathSelector("$.list[*].title").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> times = new JsonPathSelector("$.list[*].time").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date today = new Date();
        String todayStr = format.format(today);
        today = format.parse(todayStr.substring(0, 10) + " 00:00:00");
        int count = 0;

        for (int i = 0; i < links.size(); i++) {

            String d = format.format(Long.parseLong((String) times.get(i) + "000"));
            Date newsTime = new Date();
            try {
                newsTime = format.parse(d);
            } catch (ParseException e) {

                e.printStackTrace();
            }
            if (today.after(newsTime)) continue;
            page.addTargetRequest(new Request(links.get(i)).putExtra("title", titles.get(i)).putExtra("tag", tags.get(i)).putExtra("time", newsTime));
            count++;
        }
        logger.debug(String.format("【%s】%d条记录加入任务队列", getSourceFromPage(page), count));
        page.setSkip(true);
    }


    /**
     * 筛选出指定日期的文章
     *
     * @param page
     * @param date
     * @throws ParseException
     */
    public void initProcessor(Page page, String date) throws ParseException {

        if (page.getUrl().toString().indexOf(sinaType_1) != -1) {
            mixNews(page, date);
        } else if (page.getUrl().toString().indexOf(sinaType_2) != -1) {
            rollNews(page, date);
        } else if (page.getUrl().toString().indexOf("others") != -1) {
            others(page, date);
        }
    }


    /**
     * 解析部分
     *
     * @param page
     */
    private void contentProcessor(Page page) {

        page.putField("main_title", page.getHtml().$("main-title").toString());
        page.putField("subtitle", "");
        page.putField("article", page.getHtml().smartContent());
        page.putField("url", page.getUrl().toString());
        page.putField("date", (Date) page.getRequest().getExtra("time"));
    }


    /**
     * 用于判断Links是否
     *
     * @param date
     * @param matchStr
     * @return
     */
    public boolean reg(String date, String matchStr) {

        if (matchStr.indexOf(date) != -1)
            return true;
        return false;
    }


    /**
     * 其他新闻页面处理规则
     *
     * @param page
     * @param date
     */
    private void others(Page page, String date) {
    }

    /**
     * 滚动累新闻页面处理规则
     *
     * @param page
     * @param date
     */
    private void rollNews(Page page, String date) {
        List<String> links = new JsonPathSelector("$.list[*].url").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> tags = new JsonPathSelector("$.list[*].channel.title").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> titles = new JsonPathSelector("$.list[*].title").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> times = new JsonPathSelector("$.list[*].time").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));

        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        int count = 0;
        for (int i = 0; i < links.size(); i++) {
            //如果是指定日期的新闻，则加进目标请求中
            if (reg(date, links.get(i))) {
                String d = format.format(Long.parseLong((String) times.get(i) + "000"));
                Date newsTime = new Date();
                try {
                    newsTime = format.parse(d);
                } catch (ParseException e) {
                    e.printStackTrace();
                }
                page.addTargetRequest(new Request(links.get(i)).putExtra("title", titles.get(i)).putExtra("tag", tags.get(i)).putExtra("time", newsTime));
                count++;
            }
        }
        logger.info(String.format("【%s】%d条记录加入任务队列，页面类型是roll", getSourceFromPage(page), count));
        page.setSkip(true);
    }

    /**
     * 混合类新闻页面URL的提取规则
     *
     * @param page
     * @param date
     */
    private void mixNews(Page page, String date) throws ParseException {
        //预处理page
        String json = page.getRawText();
        json = ConvertUtils.clearUnicode(json);
        json = json.replaceAll("\\\\", "");
        json = json.replaceAll("\"\"\"\"", "\"\"");
        json = json.replaceAll("\"\\[", "[").replaceAll("\\]\"", "]");
        json = json.substring(json.indexOf("{\"result\""), json.indexOf(");}catch(e){};"));
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        //提取页面URLs
        List<String> links = new LinkedList<>();
        String re1 = ".*?";    // Non-greedy match on filler
        String re2 = "((?:http|https)(?::\\/{2}[\\w]+)(?:[\\/|\\.]?)(?:[^\\s\"]*))";    // HTTP URL 1

        Pattern p = Pattern.compile(re1 + re2, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
        Matcher m = p.matcher(json);
        while (m.find()) {
            //获取 字符串
            links.add(m.group());
        }


        int count = 0;
        for (int i = 0; i < links.size(); i++) {
            Pattern p1 = Pattern.compile("[a-zA-z]+://[^\\s]*");
            Matcher m2 = p1.matcher(links.get(i));
            String str = "";
            if (m2.find()) {
                //获取 字符串
                str = m2.group(0);
            }
            //如果是指定日期的新闻，则加进目标请求中
            if (reg(date, str)) {

                page.addTargetRequest(new Request(str).putExtra("time", format.parse(date + " 00:00:00")));
                count++;
            }
        }
        logger.info(String.format("【%s】%d条记录加入任务队列,页面类型为mix", getSourceFromPage(page), count));
        page.setSkip(true);

    }
}
